package com.pig4cloud.pig.demo.Util;

import com.pig4cloud.pig.demo.Service.ArticleService;
import com.pig4cloud.pig.demo.feign.Entity.Article;
import lombok.SneakyThrows;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover;
import us.codecraft.webmagic.scheduler.QueueScheduler;
import us.codecraft.webmagic.selector.Selectable;

import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;

@Component
public class ArticleContentProcessor implements PageProcessor {
    @Value("${articleContent.url}")
    private String url;

    @Autowired
    private ArticleContentPipeline articleContentPipeline;

    @Autowired
    private ArticleService articleService;

    private String category;

    private Site site = Site.me()
            .setRetryTimes(3)
            .setSleepTime(1000)
            .setTimeOut(10000);

    @SneakyThrows
    @Override
    public void process(Page page) {
        List<Selectable> nodes = page.getHtml().css("div.article-item-wrap div.article-item").nodes();
        //Jsoup.parse(page.getHtml().css("div.article-item-c a").nodes().toString()).text();

        List<Article> articles = new ArrayList<Article>();

        for (Selectable node : nodes) {
            String img = node.css("a.article-item-l img", "src").toString();
            //获取标题
            String title = Jsoup.parse(node.css("a.article-title").toString()).text();

            //获取文章跳转链接
            String articleContentUrl = node.css("div.article-item-c a", "href").toString();
            //获取作者头像
            String avatar = node.css("div.article-item-footer a.author-img img", "src").toString();

            //获取作者名称
            String name = node.css("div.article-item-footer a.article-author", "text").toString();

            //获取文章创建时间
            String createTime = node.css("div.article-item-footer span.article-author", "text").nodes().get(0).toString();

            String articleContent = getArticleContent(articleContentUrl);

            Article article = new Article();
            article.setTitle(title);
            article.setPerson(name);
            article.setContent(articleContent);
            article.setPicUrl(img);
            //article.setPublishTime(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").parse(createTime));


            articles.add(article);
        }
        page.putField("content", articles);
        //通过url获取文章的分类
        page.putField("category", page.getUrl().toString().split("=")[1]);
    }

    public void process(String category) {
        this.category = category;
        Spider.create(new ArticleContentProcessor())
                .addUrl(url + category)
                //BloomFilter去重
                .setScheduler(new QueueScheduler().setDuplicateRemover(new BloomFilterDuplicateRemover(100000)))
                .thread(5)  //5个线程
                .addPipeline(articleContentPipeline)
                .run();
    }

    private String getArticleContent(String Contenturl) throws Exception {
        url = "https://www.imooc.com/" + Contenturl;
        Document doc = Jsoup.connect(url).get();

        return doc.select("div.detail-content-wrap div.detail-content div:lt(1)").toString();
    }

    @Override
    public Site getSite() {
        return site;
    }
}
